package com.wzgl.core.webmagic.processor;

import java.io.Serializable;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.select.Elements;
import org.quartz.Job;
import org.quartz.JobExecutionContext;
import org.quartz.JobExecutionException;

import com.wzgl.core.CacheConsts;
import com.wzgl.core.model.Tgrid;
import com.wzgl.core.server.FinishurlService;
import com.wzgl.core.utils.WZGLUtils;
import com.wzgl.core.vo.GameSkyVo;
import com.wzgl.core.webmagic.MagicUtils;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;


/**
 * 网易官方 --每天爬取一页中的第一篇
 * @author Administrator
 *
 */
public class Game163Processor extends GameSkyVo implements PageProcessor,Serializable,Job{

	private static final long serialVersionUID = 1L;
	private static FinishurlService me = FinishurlService.me;
    private static Tgrid t = null;
    private static Spider s ;
    private int size = 1;
    private StringBuffer sb = new StringBuffer();
    private Site site = Site.me().setRetryTimes(1).setSleepTime(1000).addHeader("Accept-Encoding", "/");
    private String pic_ = "http://game.163.com";
	private GameSkyVo tf ;
	
	@Override
	public void execute(JobExecutionContext context) throws JobExecutionException {
		s = Spider.create(new Game163Processor()).addUrl("http://game.163.com/news/").thread(1);    
//    	s.addPipeline(new TgridsPipeLine());
//    	s.getStatus();
    	s.run();
    	System.out.println("网易官方-------------------------执行完毕");
	}
	
	public static void main(String[] args) {
		s = Spider.create(new Game163Processor()).addUrl("http://game.163.com/news/").thread(1);    
//    	s.addPipeline(new TgridsPipeLine());
//    	s.getStatus();
    	s.run();
    	System.out.println("网易官方-------------------------执行完毕");
	}

	@Override
	public void process(Page page) {
		 if(page.getUrl().regex("http://game.163.com/news/2017").match()){
    		 //在列表结果集中拿出信息
    		 //Tgrid t = Redis.use(CacheConsts.CACHE_MAGIC_GIRD_NAME).get(page.getUrl());
    		 //Tgrid t = (Tgrid) WZGLUtils.getCache(CacheConsts.CACHE_MAGIC_GIRD_NAME, page.getUrl());
			 //来源 
    		 t.setFfrom("网易网络");
    		 
    		 //下载一张图片  作为封面
    		 String src = "";
    		 String c_url = "";
    		 List<String> pic_urls = page.getHtml().xpath("//div[@class='artText']"
     		 		+ "/p[@style='text-align: center;']/img").all();
    		 Document docList = Jsoup.parse(pic_urls.get(0)); 
    		 if(docList != null){
    			 Elements liTag=docList.getElementsByTag("img");
        		 src = liTag.get(0).attr("src");
        		 c_url = MagicUtils.downloadPic(src);
    		 }
    		 
    		 //正文
    		 String  cont = "";
    		 String pic_text = page.getHtml().xpath("//div[@class='artText']").toString();
    		 cont = pic_text.substring(pic_text.indexOf("<p>"), pic_text.lastIndexOf("/>"));
    		 System.out.println("cont:  "+cont+"\n");
    		 t.setFhtml(cont);
    		 t.setFcoverImg(c_url);
    		 
//    		 //通过url缓存此对象
//    		 String cachename = CacheConsts.CACHE_MAGIC_GIRD_NAME;
//			 String name = cachename+":"+page.getUrl();
//			 WZGLUtils.setCache(cachename, name, t);
//			 //更新爬取日志
//    		 sb.append("读取第"+size+"详情.................").append("\n\r");
//    		 String log = (String) WZGLUtils.getCache(CacheConsts.CACHE_LOG_MAGIC_NAME,"爬取日志:"+pic_);
//    		 log += sb.toString();
//    		 String log_cache = CacheConsts.CACHE_LOG_MAGIC_NAME;
//    		 String log_name = CacheConsts.CACHE_LOG_MAGIC_NAME+":"+pic_;
//    		 WZGLUtils.setCache(log_cache, log_name, log);
    		 
    		 size++; 
		 }else if(page.getUrl().regex("http://game.163.com/news/index_").match()){
			 List<String> page_1= page.getHtml().xpath("//div[@class='news-list']/ul[@class='isotope']/li").all();
	   		 int num1=1;
	       	 for(String li : page_1){
	       		t = new Tgrid();  		 
	       		Document docList = Jsoup.parse(li);
	       		//网页详情
	       		String href = docList.select("a").attr("href");
	       		//列表中显示的图片地址
	       		//String img = docList.getElementsByClass("img").select("img").attr("src");
	       		//列表标题
	       		String title = docList.select("a").select("span").text();
	       		//列表摘要
	       		String desc = docList.getElementsByClass("remark").text();
	       		//列表发布时间
	       		String time = docList.getElementsByClass("time").text();
	       		//是否已经爬取过
	       		System.out.println("地址："+href);
	       		System.out.println("标题："+title+"\n");
	       		
	       		page.setSkip(true);
	       		page.addTargetRequest(href);
	       		num1++;
	       	 }
		 }else{
			 List<String> page_1= page.getHtml().xpath("//div[@class='news-list']/ul[@class='isotope']/li").all();
	   		 int num1=1;
	   		 for(int i = 0;i<1;i++){
	       		t = new Tgrid();  	
	       		String li = page_1.get(i);	 
	       		Document docList = Jsoup.parse(li);
	       		//网页详情
	       		String href = docList.select("a").attr("href");
	       		//列表中显示的图片地址
	       		//String img = docList.getElementsByClass("img").select("img").attr("src");
	       		//列表标题
	       		String title = docList.select("a").select("span").text();
	       		//列表摘要
	       		String desc = docList.getElementsByClass("remark").text();
	       		//列表发布时间
	       		String time = docList.getElementsByClass("time").text();
	       		//是否已经爬取过
	       		System.out.println("地址："+href);
	       		System.out.println("标题："+title+"\n");
	       		String cn = CacheConsts.CACHE_MAGIC_GIRD_NAME; 
	       		String key = CacheConsts.CACHE_MAGIC_GIRD_NAME+":"+href;
//	       		if(!WZGLUtils.exists(cn, key)){
//	       			tf = new GameSkyVo();
//	       			tf.setFdescribe("网易游戏-新闻咨询");
//	       			tf.setFmzgicId("afadfafaf1123121");
//	       			//tf.setFtype("10001");
//	       			//保存到对象
//			       	t.setFtitle(title);t.setFdate(new Date());t.setFtitle(title);
//			       	t.setFaddress(href);t.setFgetDate(time);t.setFdescribe(tf.getFdescribe());
//			       	t.setFtype(tf.getFtype());t.setFmzgicId(tf.getFmzgicId());t.setFstatus(0);
////			       	//把爬取了的url放入缓存
////			       	String cachename = CacheConsts.CACHE_MAGIC_GIRD_NAME;
////					String name = cachename+":"+href;
////					WZGLUtils.setCache(cachename, name, t); 
//					//记录爬取日志
//	        		sb.append("第"+num1+"文章开始读取----------------->\r");
//	        		sb.append("第"+num1+"标题:"+title).append("\r");
//	        		sb.append("第"+num1+"文章详情URL:"+href).append("\r");
//	        		sb.append("第"+num1+"文章时间:"+time).append("\r");
//	        		sb.append("第"+num1+"文章开始结束----------------->").append("\r");
//	        		//记录日志加入缓存
////	        		String log = Redis.use(CacheConsts.CACHE_LOG_MAGIC_NAME).get(pic_);
////	        		log += sb.toString();
////	        		String log_cache = CacheConsts.CACHE_LOG_MAGIC_NAME;
////	        		String log_name = CacheConsts.CACHE_LOG_MAGIC_NAME+":"+pic_;
////	        		WZGLUtils.setCache(log_cache, log_name, log);
//	        		//把爬取了的url加入数据库
////	        		TfinishUrl tf = new TfinishUrl();
////	        		tf.setFurl(href);
////	        		tf.setFdate(new Date());
////	        		me.save(tf);
//		       	    page.setSkip(true);
//		       		//获取文章正文
//		       		page.addTargetRequest(href);
//	       		}
	       		
	       		page.setSkip(true);
	       		page.addTargetRequest(href);
	       		num1++;
	       	 }
	       	 
	       	//控制获取分页 
//	       	List<String> hrefs = page.getHtml().xpath("//div[@class='news-list']/p[@class='paging']/a").all();
//       		for(int i = 0;i<hrefs.size()-6;i++){
//       			String a = hrefs.get(i);
//       			Document docList = Jsoup.parse(a);
//	       		//获取链接
//	       		String href = docList.select("a").attr("href");
//	       		System.out.println("获取分页："+href);
//	       		page.setSkip(true);
//	       		page.addTargetRequest(href);
//	       		num1++;
//       		}
		 }
	}

	@Override
	public Site getSite() {
		return site;
	}
}
